package fetcher

import (
	"net/http"
	"fmt"
	"golang.org/x/text/transform"
	"io/ioutil"
	"clawer/common/utils"
	"bufio"
	"time"
)

const (
	MOCK_USER_AGENT = "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1"
)

var rateLimiter = time.Tick(100 * time.Millisecond)

type Service interface {
	Fetch(string) ([]byte, error)
}

type service struct {
}

func NewService() Service {
	return &service{}
}

func (s *service) Fetch(url string) ([]byte, error) {
	<-rateLimiter
	request, _ := http.NewRequest(http.MethodGet, url, nil)
	request.Header.Add("User-Agent", MOCK_USER_AGENT)
	resp, err := http.DefaultClient.Do(request)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil,
			fmt.Errorf("Wrong status code: %d", resp.StatusCode)
	}
	//自动检测编码格式
	bodyReader := bufio.NewReader(resp.Body)
	e := utils.DetermineEncoding(bodyReader)
	//转换为utf8编码
	utf8Reader := transform.NewReader(bodyReader,
		e.NewDecoder())
	return ioutil.ReadAll(utf8Reader)
}
